#!/usr/bin/env python 
# -*- coding: utf-8 -*- 
# @Time : 2018-06-06 10:48 
# @Author : Leo
# @File : Step4_DownloadUpdateCover.py
"""
查找DB3中封面缺失的字段，
在MySQL中查找封面url进行下载，
并更新至上一步得到的DB3中
"""

import logging
import os
import random
import time

import facade
import requests
import urllib3
from xjlibrary.our_file_dir import BaseDir, ImageFile

curPath = BaseDir.get_file_dir_absolute(__file__)
TopPath = BaseDir.get_upper_dir(curPath, -2)
dirPath = BaseDir.get_new_path(TopPath, "download", "TandBook", "download", "cover")

logger = facade.get_streamlogger()
configfile = BaseDir.get_new_path(curPath, "db.ini")
mysqlutils = facade.MysqlUtiles(configfile, "db1", logger=logger)

HEADERS = {
    'Host': 'images.tandf.co.uk',
    'Referer': 'https://www.taylorfrancis.com/',
    'origin': 'https://www.taylorfrancis.com',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
}


# 装饰器：请求失败重试
def retry(func):
    RETRIES = 0
    count = {'num': RETRIES}

    def wrapper(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except Exception as err:
            if count['num'] < 3:  # 重新请求次数
                count['num'] += 1
                logging.warning('The %s Retrying...' % count['num'])
                return wrapper(*args, **kwargs)
            else:
                raise Exception(err)

    return wrapper


# 获取DB3中不存在cover的更新任务
def getNoCoverTask():
    # Select_Sql = "SELECT rawid FROM modify_title_info_zt WHERE cover IS NULL LIMIT 20"
    Select_Sql = "SELECT rawid FROM allfields WHERE cover IS NULL"
    result = mysqlutils.SelectFromDB(Select_Sql)
    if result:
        without_cover_set = (each for each in set(map(lambda each: each[0], result)))
        return without_cover_set
    else:
        return None


# 从MySQL数据库获取封面URL
def getUrlFromDB(isbn):
    """
    :param mysql_conn: MySQL连接
    :param isbn: 书籍isbn号
    :return: 所有封面的url，可用的url
    """
    Sql = "SELECT isbn13, url, ava_url FROM cover WHERE isbn13='%s' AND failcount < 2" % isbn
    result = mysqlutils.SelectFromDB(Sql)
    if not result:
        return '', ''
    all_url, ava_url = result[0][1], result[0][2]
    return all_url, ava_url


# 检查是否已被下载
def checkDownFromDB(isbn):
    """
    :param mysql_conn: MySQL连接
    :param isbn: 书籍isbn号
    :return: 所有封面的url，可用的url
    """
    Sql = "SELECT isbn13 FROM cover WHERE isbn13='%s' AND stat=1" % isbn
    result = mysqlutils.SelectFromDB(Sql)
    return result


# 更新数据库中cover表
def updateDBCover(isbn, status, new_ava_url):
    if status:
        Sql = "UPDATE cover SET ava_url='%s',stat=1 WHERE isbn13='%s'" % (new_ava_url, isbn)
    else:
        # Sql = "INSERT IGNORE INTO cover(isbn13) VALUES('%s')" % isbn
        Sql = "UPDATE cover SET failcount= failcount + 1 WHERE isbn13='%s'" % isbn
    mysqlutils.ExeSqlToDB(Sql)


# 获取当天日期
def getTodayDate():
    return time.strftime('%Y%m%d', time.localtime(time.time()))


# 更新DB3中的cover字段
def updateDBAllFieldCover(isbn):
    cover_field = '/smartlib/tandfbook/' + isbn[:7] + '/' + isbn + '.jpg'
    batch = getTodayDate() + '00'  # 设置批次
    Sql = "UPDATE allfields SET cover='%s', batch='%s' WHERE rawid='%s'" % (cover_field, batch, isbn)
    mysqlutils.ExeSqlToDB(Sql)


# 获取封面保存路径
def getCoverPath(isbn, url):
    dir_path = os.path.join(dirPath, isbn[:7])
    if not os.path.exists(dir_path):
        os.makedirs(dir_path)
    # if url.endswith('.tif'):  # 封面格式为tif时
    #     cover_name = isbn + '.tif'
    # elif url.endswith('.jpg'):
    #     cover_name = isbn + '.jpg'
    # else:
    #     cover_name = isbn + '.unknown'
    cover_name = isbn + '.jpg'
    cover_path = os.path.join(dir_path, cover_name)
    logging.info('Save cover to <%s>' % cover_path)
    return cover_path


@retry
def reqUrl(sn, url):
    time.sleep(random.random())
    resp = sn.get(url=url, timeout=5)
    print(resp.status_code)
    assert resp.status_code == 200
    return resp


# 根据URL下载封面
def downloadCover(session, isbn, all_url, ava_url):
    """
    :param session: 会话连接
    :param isbn: 对应的isbn
    :param all_url: 所有的封面url字符串
    :param ava_url: 验证可用的封面url
    :return: 封面保存成功返回True,以及可用ava_url,反之为False
    """
    if ava_url:
        try:
            print('[1] >> ' + ava_url)
            resp = reqUrl(session, ava_url)
            cover_path = getCoverPath(isbn=isbn, url=ava_url)
            print(cover_path)
            ImageFile.img_to_format_jpg(resp.content, cover_path)
            return True, ava_url
        except:
            pass
    else:
        all_cover_list = all_url.split('**')
        if all_cover_list[0]:
            for each_url in all_cover_list:
                try:
                    print('[2] >> ' + each_url)
                    resp = reqUrl(session, each_url)
                    print("success request image url")
                    cover_path = getCoverPath(isbn=isbn, url=each_url)
                    print(cover_path)
                    ImageFile.img_to_format_jpg(resp.content, cover_path)
                    return True, each_url
                except:
                    pass
    return False, ''


def start():
    session = requests.Session()
    # requests.packages.urllib3.disable_warnings()
    # 帮助快速禁用urllib3的警告
    urllib3.disable_warnings()
    session.headers = HEADERS
    # 关闭ssl验证
    session.verify = False
    logging.warning('Set headers:True, verify:False, disable_warning:True')
    isbn_set_gen = getNoCoverTask()  # 查询得到的不存在cover的生成器
    logging.info('Get database field success !')
    cover_count, check_count = 0, 0
    if isbn_set_gen is not None:
        for each_isbn in isbn_set_gen:
            result = checkDownFromDB(each_isbn)
            # 表示已经下载，直接更新到all表即可
            if result:
                updateDBAllFieldCover(result[0][0])
                continue
            all_url, ava_url = getUrlFromDB(each_isbn)  # 所有封面url，验证可用的url
            download_status, new_ava_url = downloadCover(session, each_isbn, all_url, ava_url)
            check_count += 1
            if download_status:
                cover_count += 1
                updateDBAllFieldCover(each_isbn)  # 更新到DB3文件中
            logging.info('Current isbn:%s, Check:%s, Download:%s, Success:%s' % (
                each_isbn, check_count, cover_count, str(download_status)))
            updateDBCover(isbn=each_isbn, status=download_status, new_ava_url=new_ava_url)


if __name__ == '__main__':
    start()
